{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "98bc3fa2",
   "metadata": {},
   "source": [
    "[ref - How to Develop LSTM Models for Time Series Forecasting](https://machinelearningmastery.com/how-to-develop-lstm-models-for-time-series-forecasting/)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dec5c2f1",
   "metadata": {},
   "source": [
    "# <font color=#FF6347>一、Univariate LSTM Models</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "8d729297",
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "1e9f6780",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[10 20 30] 40\n",
      "[20 30 40] 50\n",
      "[30 40 50] 60\n",
      "[40 50 60] 70\n",
      "[50 60 70] 80\n",
      "[60 70 80] 90\n"
     ]
    }
   ],
   "source": [
    "from numpy import array\n",
    " \n",
    "# split a univariate sequence into samples\n",
    "def split_sequence(sequence, n_steps):\n",
    "\tX, y = list(), list()\n",
    "\tfor i in range(len(sequence)):\n",
    "\t\t# find the end of this pattern\n",
    "\t\tend_ix = i + n_steps\n",
    "\t\t# check if we are beyond the sequence\n",
    "\t\tif end_ix > len(sequence)-1:\n",
    "\t\t\tbreak\n",
    "\t\t# gather input and output parts of the pattern\n",
    "\t\tseq_x, seq_y = sequence[i:end_ix], sequence[end_ix]\n",
    "\t\tX.append(seq_x)\n",
    "\t\ty.append(seq_y)\n",
    "\treturn array(X), array(y)\n",
    " \n",
    "# define input sequence\n",
    "raw_seq = [10, 20, 30, 40, 50, 60, 70, 80, 90]\n",
    "\n",
    "# choose a number of time steps\n",
    "n_steps = 3\n",
    "\n",
    "# split into samples\n",
    "X, y = split_sequence(raw_seq, n_steps)\n",
    "\n",
    "# summarize the data\n",
    "for i in range(len(X)):\n",
    "\tprint(X[i], y[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "9f01a35e",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import LSTM\n",
    "from tensorflow.keras.layers import Dense"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "efd92dc5",
   "metadata": {},
   "source": [
    "# Vanilla LSTM\n",
    "training data to have the dimensions or shape:\n",
    "\n",
    "[samples, timesteps, features]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "ae26fa3d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6, 3, 1)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_features = 1\n",
    "X = X.reshape((X.shape[0], X.shape[1], n_features))\n",
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "e923204d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:8 out of the last 8 calls to <function Model.make_predict_function.<locals>.predict_function at 0x000001C55EB93F78> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "[[102.5332]]\n"
     ]
    }
   ],
   "source": [
    "# define model\n",
    "model = Sequential()\n",
    "model.add(LSTM(50, activation='relu', input_shape=(n_steps, n_features)))\n",
    "model.add(Dense(1))   # output\n",
    "model.compile(optimizer='adam', loss='mse')\n",
    "\n",
    "# fit model\n",
    "model.fit(X, y, epochs=200, verbose=0)\n",
    "\n",
    "# demonstrate prediction\n",
    "x_input = array([70, 80, 90])\n",
    "x_input = x_input.reshape((1, n_steps, n_features))\n",
    "yhat = model.predict(x_input, verbose=0)\n",
    "print(yhat)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dd174c84",
   "metadata": {},
   "source": [
    "## Stacked LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "3796a162",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:9 out of the last 9 calls to <function Model.make_predict_function.<locals>.predict_function at 0x000001C5621974C8> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "[[103.76633]]\n"
     ]
    }
   ],
   "source": [
    "# define model\n",
    "model = Sequential()\n",
    "model.add(LSTM(50, activation='relu', return_sequences=True, input_shape=(n_steps, n_features)))\n",
    "model.add(LSTM(50, activation='relu'))\n",
    "model.add(Dense(1))\n",
    "model.compile(optimizer='adam', loss='mse')\n",
    "\n",
    "# fit model\n",
    "model.fit(X, y, epochs=200, verbose=0)\n",
    "\n",
    "# demonstrate prediction\n",
    "x_input = array([70, 80, 90])\n",
    "x_input = x_input.reshape((1, n_steps, n_features))\n",
    "yhat = model.predict(x_input, verbose=0)\n",
    "print(yhat)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d2ada80a",
   "metadata": {},
   "source": [
    "## Bidirectional LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "0c3adaaf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:10 out of the last 10 calls to <function Model.make_predict_function.<locals>.predict_function at 0x000001C55ECD7798> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "[[101.3157]]\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.keras.layers import Bidirectional\n",
    "\n",
    "# define model\n",
    "model = Sequential()\n",
    "model.add(Bidirectional(LSTM(50, activation='relu'), input_shape=(n_steps, n_features)))\n",
    "model.add(Dense(1))\n",
    "model.compile(optimizer='adam', loss='mse')\n",
    "\n",
    "# fit model\n",
    "model.fit(X, y, epochs=200, verbose=0)\n",
    "\n",
    "# demonstrate prediction\n",
    "x_input = array([70, 80, 90])\n",
    "x_input = x_input.reshape((1, n_steps, n_features))\n",
    "yhat = model.predict(x_input, verbose=0)\n",
    "print(yhat)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c014c468",
   "metadata": {},
   "source": [
    "## CNN LSTM\n",
    "The first step is to split the input sequences into subsequences that can be processed by the CNN model. For example, we can first split our univariate time series data into input/output samples with four steps as input and one as output. Each sample can then be split into two sub-samples, each with two time steps. The CNN can interpret each subsequence of two time steps and provide a time series of interpretations of the subsequences to the LSTM model to process as input.\n",
    "\n",
    "We can parameterize this and define the number of subsequences as n_seq and the number of time steps per subsequence as n_steps. The input data can then be reshaped to have the required structure:\n",
    "\n",
    "[samples, subsequences, timesteps, features]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "2a43b7a0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[10 20 30 40] 50\n",
      "[20 30 40 50] 60\n",
      "[30 40 50 60] 70\n",
      "[40 50 60 70] 80\n",
      "[50 60 70 80] 90\n"
     ]
    }
   ],
   "source": [
    "# choose a number of time steps\n",
    "n_steps = 4\n",
    "\n",
    "# split into samples\n",
    "X, y = split_sequence(raw_seq, n_steps)\n",
    "\n",
    "for i in range(len(X)):\n",
    "    print(X[i], y[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "cfe8a973",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5, 2, 2, 1)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# reshape from [samples, timesteps] into [samples, subsequences, timesteps, features]\n",
    "n_features = 1\n",
    "n_seq = 2\n",
    "n_steps = 2\n",
    "X = X.reshape((X.shape[0], n_seq, n_steps, n_features))\n",
    "X.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d5db20d2",
   "metadata": {},
   "source": [
    "[How to Use the TimeDistributed Layer in Keras](https://machinelearningmastery.com/timedistributed-layer-for-long-short-term-memory-networks-in-python/)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "bbfb95be",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.layers import Flatten\n",
    "from tensorflow.keras.layers import TimeDistributed\n",
    "from tensorflow.keras.layers import Conv1D\n",
    "from tensorflow.keras.layers import MaxPooling1D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "d30e043c",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x000001C5632EA708> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "[[101.033264]]\n"
     ]
    }
   ],
   "source": [
    "# define model\n",
    "model = Sequential()\n",
    "model.add(TimeDistributed(Conv1D(filters=64, kernel_size=1, activation='relu'), input_shape=(None, n_steps, n_features)))\n",
    "model.add(TimeDistributed(MaxPooling1D(pool_size=2)))\n",
    "model.add(TimeDistributed(Flatten()))\n",
    "model.add(LSTM(50, activation='relu'))\n",
    "model.add(Dense(1))\n",
    "model.compile(optimizer='adam', loss='mse')\n",
    "\n",
    "# fit model\n",
    "model.fit(X, y, epochs=500, verbose=0)\n",
    "\n",
    "# demonstrate prediction\n",
    "x_input = array([60, 70, 80, 90])\n",
    "x_input = x_input.reshape((1, n_seq, n_steps, n_features))\n",
    "yhat = model.predict(x_input, verbose=0)\n",
    "print(yhat)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0127633c",
   "metadata": {},
   "source": [
    "## ConvLSTM\n",
    "\n",
    "The ConvLSTM was developed for reading two-dimensional spatial-temporal data, but can be adapted for use with univariate time series forecasting.\n",
    "\n",
    "[samples, timesteps, rows, columns, features]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "5d9a2034",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5, 2, 1, 2, 1)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# choose a number of time steps\n",
    "n_steps = 4\n",
    "\n",
    "# split into samples\n",
    "X, y = split_sequence(raw_seq, n_steps)\n",
    "\n",
    "# reshape from [samples, timesteps] into [samples, timesteps, rows, columns, features]\n",
    "n_features = 1\n",
    "n_seq = 2\n",
    "n_steps = 2\n",
    "X = X.reshape((X.shape[0], n_seq, 1, n_steps, n_features))\n",
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "3c34e780",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.layers import ConvLSTM2D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "c5d19670",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x000001C5581C8A68> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "[[103.019196]]\n"
     ]
    }
   ],
   "source": [
    "# define model\n",
    "model = Sequential()\n",
    "model.add(ConvLSTM2D(filters=64, kernel_size=(1,2), activation='relu', input_shape=(n_seq, 1, n_steps, n_features)))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(1))\n",
    "model.compile(optimizer='adam', loss='mse')\n",
    "\n",
    "# fit model\n",
    "model.fit(X, y, epochs=500, verbose=0)\n",
    "\n",
    "# demonstrate prediction\n",
    "x_input = array([60, 70, 80, 90])\n",
    "x_input = x_input.reshape((1, n_seq, 1, n_steps, n_features))\n",
    "yhat = model.predict(x_input, verbose=0)\n",
    "print(yhat)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a8c945ce",
   "metadata": {},
   "source": [
    "# <font color=#FF6347>二、Multivariate LSTM Models</font>\n",
    "data:<br>\n",
    "- Multiple Input Series.\n",
    "- Multiple Parallel Series"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a2e84756",
   "metadata": {},
   "source": [
    "### Multiple Input Series."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "e5edfc06",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(7, 3, 2) (7,)\n",
      "[[10 15]\n",
      " [20 25]\n",
      " [30 35]] 65 ====\n",
      "[[20 25]\n",
      " [30 35]\n",
      " [40 45]] 85 ====\n",
      "[[30 35]\n",
      " [40 45]\n",
      " [50 55]] 105 ====\n",
      "[[40 45]\n",
      " [50 55]\n",
      " [60 65]] 125 ====\n",
      "[[50 55]\n",
      " [60 65]\n",
      " [70 75]] 145 ====\n",
      "[[60 65]\n",
      " [70 75]\n",
      " [80 85]] 165 ====\n",
      "[[70 75]\n",
      " [80 85]\n",
      " [90 95]] 185 ====\n"
     ]
    }
   ],
   "source": [
    "# multivariate data preparation\n",
    "from numpy import array\n",
    "from numpy import hstack\n",
    "\n",
    "# split a multivariate sequence into samples\n",
    "def split_sequences(sequences, n_steps):\n",
    "\tX, y = list(), list()\n",
    "\tfor i in range(len(sequences)):\n",
    "\t\t# find the end of this pattern\n",
    "\t\tend_ix = i + n_steps\n",
    "\t\t# check if we are beyond the dataset\n",
    "\t\tif end_ix > len(sequences):\n",
    "\t\t\tbreak\n",
    "\t\t# gather input and output parts of the pattern\n",
    "\t\tseq_x, seq_y = sequences[i:end_ix, :-1], sequences[end_ix-1, -1]\n",
    "\t\tX.append(seq_x)\n",
    "\t\ty.append(seq_y)\n",
    "\treturn array(X), array(y)\n",
    "\n",
    "# define input sequence\n",
    "in_seq1 = array([10, 20, 30, 40, 50, 60, 70, 80, 90])\n",
    "in_seq2 = array([15, 25, 35, 45, 55, 65, 75, 85, 95])\n",
    "out_seq = array([in_seq1[i]+in_seq2[i] for i in range(len(in_seq1))])\n",
    "\n",
    "# convert to [rows, columns] structure\n",
    "in_seq1 = in_seq1.reshape((len(in_seq1), 1))\n",
    "in_seq2 = in_seq2.reshape((len(in_seq2), 1))\n",
    "out_seq = out_seq.reshape((len(out_seq), 1))\n",
    "\n",
    "# horizontally stack columns\n",
    "dataset = hstack((in_seq1, in_seq2, out_seq))\n",
    "\n",
    "# choose a number of time steps\n",
    "n_steps = 3\n",
    "\n",
    "# convert into input/output\n",
    "X, y = split_sequences(dataset, n_steps)\n",
    "print(X.shape, y.shape)\n",
    "\n",
    "# summarize the data\n",
    "for i in range(len(X)):\n",
    "\tprint(X[i], y[i], '====')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "48c7448a",
   "metadata": {},
   "source": [
    "X component has a three-dimensional structure.\n",
    "\n",
    "The first dimension is the number of samples, in this case 7. The second dimension is the number of time steps per sample, in this case 3, the value specified to the function. Finally, the last dimension specifies the number of parallel time series or the number of variables, in this case 2 for the two parallel series."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "2555df06",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "feature nums:  2\n",
      "x input:  [[[ 80  85]\n",
      "  [ 90  95]\n",
      "  [100 105]]]\n",
      "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x000001C55FE46828> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "[[205.90128]]\n"
     ]
    }
   ],
   "source": [
    "# the dataset knows the number of features, e.g. 2\n",
    "n_features = X.shape[2]\n",
    "print('feature nums: ', n_features)\n",
    "\n",
    "# define model\n",
    "model = Sequential()\n",
    "model.add(LSTM(50, activation='relu', input_shape=(n_steps, n_features)))\n",
    "model.add(Dense(1))\n",
    "model.compile(optimizer='adam', loss='mse')\n",
    "\n",
    "# fit model\n",
    "model.fit(X, y, epochs=200, verbose=0)\n",
    "\n",
    "# demonstrate prediction\n",
    "x_input = array([[80, 85], [90, 95], [100, 105]])\n",
    "x_input = x_input.reshape((1, n_steps, n_features))\n",
    "print('x input: ', x_input)\n",
    "\n",
    "yhat = model.predict(x_input, verbose=0)\n",
    "print(yhat)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6ad0fa95",
   "metadata": {},
   "source": [
    "### Multiple Parallel Series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "be336dcb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(6, 3, 3) (6, 3)\n",
      "[[10 15 25]\n",
      " [20 25 45]\n",
      " [30 35 65]] [40 45 85]\n",
      "[[20 25 45]\n",
      " [30 35 65]\n",
      " [40 45 85]] [ 50  55 105]\n",
      "[[ 30  35  65]\n",
      " [ 40  45  85]\n",
      " [ 50  55 105]] [ 60  65 125]\n",
      "[[ 40  45  85]\n",
      " [ 50  55 105]\n",
      " [ 60  65 125]] [ 70  75 145]\n",
      "[[ 50  55 105]\n",
      " [ 60  65 125]\n",
      " [ 70  75 145]] [ 80  85 165]\n",
      "[[ 60  65 125]\n",
      " [ 70  75 145]\n",
      " [ 80  85 165]] [ 90  95 185]\n"
     ]
    }
   ],
   "source": [
    "# multivariate output data prep\n",
    "from numpy import array\n",
    "from numpy import hstack\n",
    "\n",
    "# split a multivariate sequence into samples\n",
    "def split_sequences(sequences, n_steps):\n",
    "\tX, y = list(), list()\n",
    "\tfor i in range(len(sequences)):\n",
    "\t\t# find the end of this pattern\n",
    "\t\tend_ix = i + n_steps\n",
    "\t\t# check if we are beyond the dataset\n",
    "\t\tif end_ix > len(sequences)-1:\n",
    "\t\t\tbreak\n",
    "\t\t# gather input and output parts of the pattern\n",
    "\t\tseq_x, seq_y = sequences[i:end_ix, :], sequences[end_ix, :]\n",
    "\t\tX.append(seq_x)\n",
    "\t\ty.append(seq_y)\n",
    "\treturn array(X), array(y)\n",
    "\n",
    "# define input sequence\n",
    "in_seq1 = array([10, 20, 30, 40, 50, 60, 70, 80, 90])\n",
    "in_seq2 = array([15, 25, 35, 45, 55, 65, 75, 85, 95])\n",
    "out_seq = array([in_seq1[i]+in_seq2[i] for i in range(len(in_seq1))])\n",
    "\n",
    "# convert to [rows, columns] structure\n",
    "in_seq1 = in_seq1.reshape((len(in_seq1), 1))\n",
    "in_seq2 = in_seq2.reshape((len(in_seq2), 1))\n",
    "out_seq = out_seq.reshape((len(out_seq), 1))\n",
    "\n",
    "# horizontally stack columns\n",
    "dataset = hstack((in_seq1, in_seq2, out_seq))\n",
    "\n",
    "# choose a number of time steps\n",
    "n_steps = 3\n",
    "\n",
    "# convert into input/output\n",
    "X, y = split_sequences(dataset, n_steps)\n",
    "print(X.shape, y.shape)\n",
    "\n",
    "# summarize the data\n",
    "for i in range(len(X)):\n",
    "\tprint(X[i], y[i])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9e81f043",
   "metadata": {},
   "source": [
    "The shape of X is three-dimensional, including the number of samples (6), the number of time steps chosen per sample (3), and the number of parallel time series or features (3).\n",
    "\n",
    "The shape of y is two-dimensional as we might expect for the number of samples (6) and the number of time variables per sample to be predicted (3)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "42d37678",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "feature nums:  3\n",
      "x input:  [[[ 70  75 145]\n",
      "  [ 80  85 165]\n",
      "  [ 90  95 185]]]\n",
      "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x000001C5621978B8> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "[[100.07106 105.24894 205.67851]]\n"
     ]
    }
   ],
   "source": [
    "# the dataset knows the number of features, e.g. 3\n",
    "n_features = X.shape[2]\n",
    "print('feature nums: ', n_features)\n",
    "\n",
    "# define stacked LSTM model\n",
    "model = Sequential()\n",
    "model.add(LSTM(100, activation='relu', return_sequences=True, input_shape=(n_steps, n_features)))\n",
    "model.add(LSTM(100, activation='relu'))\n",
    "model.add(Dense(n_features))\n",
    "model.compile(optimizer='adam', loss='mse')\n",
    "\n",
    "# fit model\n",
    "model.fit(X, y, epochs=400, verbose=0)\n",
    "\n",
    "# demonstrate prediction\n",
    "x_input = array([[70,75,145], [80,85,165], [90,95,185]])\n",
    "x_input = x_input.reshape((1, n_steps, n_features))\n",
    "print('x input: ', x_input)\n",
    "\n",
    "yhat = model.predict(x_input, verbose=0)\n",
    "print(yhat)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "059e36ce",
   "metadata": {},
   "source": [
    "# <font color=#FF6347>三、Multi-Step LSTM Models</font>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ae817e50",
   "metadata": {},
   "source": [
    "There are two main types of LSTM models that can be used for multi-step forecasting; they are:\n",
    "\n",
    "- Vector Output Model\n",
    "- Encoder-Decoder Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "f18ac3dc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[10 20 30] [40 50]\n",
      "[20 30 40] [50 60]\n",
      "[30 40 50] [60 70]\n",
      "[40 50 60] [70 80]\n",
      "[50 60 70] [80 90]\n"
     ]
    }
   ],
   "source": [
    "# multi-step data preparation\n",
    "from numpy import array\n",
    "\n",
    "# split a univariate sequence into samples\n",
    "def split_sequence(sequence, n_steps_in, n_steps_out):\n",
    "\tX, y = list(), list()\n",
    "\tfor i in range(len(sequence)):\n",
    "\t\t# find the end of this pattern\n",
    "\t\tend_ix = i + n_steps_in\n",
    "\t\tout_end_ix = end_ix + n_steps_out\n",
    "\t\t# check if we are beyond the sequence\n",
    "\t\tif out_end_ix > len(sequence):\n",
    "\t\t\tbreak\n",
    "\t\t# gather input and output parts of the pattern\n",
    "\t\tseq_x, seq_y = sequence[i:end_ix], sequence[end_ix:out_end_ix]\n",
    "\t\tX.append(seq_x)\n",
    "\t\ty.append(seq_y)\n",
    "\treturn array(X), array(y)\n",
    "\n",
    "# define input sequence\n",
    "raw_seq = [10, 20, 30, 40, 50, 60, 70, 80, 90]\n",
    "\n",
    "# choose a number of time steps\n",
    "n_steps_in, n_steps_out = 3, 2\n",
    "\n",
    "# split into samples\n",
    "X, y = split_sequence(raw_seq, n_steps_in, n_steps_out)\n",
    "\n",
    "# summarize the data\n",
    "for i in range(len(X)):\n",
    "    print(X[i], y[i])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "883fcb31",
   "metadata": {},
   "source": [
    "### Vector Output Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "cde3866e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x input:  [[[70]\n",
      "  [80]\n",
      "  [90]]]\n",
      "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x000001C566674A68> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "[[109.10601 132.13676]]\n"
     ]
    }
   ],
   "source": [
    "# reshape from [samples, timesteps] into [samples, timesteps, features]\n",
    "n_features = 1\n",
    "X = X.reshape((X.shape[0], X.shape[1], n_features))\n",
    "\n",
    "# define model\n",
    "model = Sequential()\n",
    "model.add(LSTM(100, activation='relu', return_sequences=True, input_shape=(n_steps_in, n_features)))\n",
    "model.add(LSTM(100, activation='relu'))\n",
    "model.add(Dense(n_steps_out))\n",
    "model.compile(optimizer='adam', loss='mse')\n",
    "\n",
    "# fit model\n",
    "model.fit(X, y, epochs=50, verbose=0)\n",
    "\n",
    "# demonstrate prediction\n",
    "x_input = array([70, 80, 90])\n",
    "x_input = x_input.reshape((1, n_steps_in, n_features))\n",
    "print('x input: ', x_input)\n",
    "\n",
    "yhat = model.predict(x_input, verbose=0)\n",
    "print(yhat)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9a5330ed",
   "metadata": {},
   "source": [
    "### Encoder-Decoder Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "cc122dc2",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.layers import RepeatVector  ##"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "3a58161b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x input:  [[[70]\n",
      "  [80]\n",
      "  [90]]]\n",
      "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x000001C56697A558> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "[[[ 97.08889 ]\n",
      "  [110.880035]]]\n"
     ]
    }
   ],
   "source": [
    "# reshape from [samples, timesteps] into [samples, timesteps, features]\n",
    "n_features = 1\n",
    "X = X.reshape((X.shape[0], X.shape[1], n_features))\n",
    "y = y.reshape((y.shape[0], y.shape[1], n_features))\n",
    "\n",
    "# define model\n",
    "model = Sequential()\n",
    "model.add(LSTM(100, activation='relu', input_shape=(n_steps_in, n_features)))\n",
    "model.add(RepeatVector(n_steps_out))\n",
    "\n",
    "model.add(LSTM(100, activation='relu', return_sequences=True))\n",
    "model.add(TimeDistributed(Dense(1)))\n",
    "model.compile(optimizer='adam', loss='mse')\n",
    "\n",
    "# fit model\n",
    "model.fit(X, y, epochs=100, verbose=0)\n",
    "\n",
    "# demonstrate prediction\n",
    "x_input = array([70, 80, 90])\n",
    "x_input = x_input.reshape((1, n_steps_in, n_features))\n",
    "print('x input: ', x_input)\n",
    "\n",
    "yhat = model.predict(x_input, verbose=0)\n",
    "print(yhat)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "72f7ffae",
   "metadata": {},
   "source": [
    "# <font color=#FF6347>四、Multivariate Multi-Step LSTM Models</font>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "29075cc4",
   "metadata": {},
   "source": [
    "- Multiple Input Multi-Step Output.\n",
    "- Multiple Parallel Input and Multi-Step Output."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7bcfaecf",
   "metadata": {},
   "source": [
    "### Multiple Input Multi-Step Output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "5e41c425",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(6, 3, 2) (6, 2)\n",
      "[[10 15]\n",
      " [20 25]\n",
      " [30 35]] [65 85]\n",
      "[[20 25]\n",
      " [30 35]\n",
      " [40 45]] [ 85 105]\n",
      "[[30 35]\n",
      " [40 45]\n",
      " [50 55]] [105 125]\n",
      "[[40 45]\n",
      " [50 55]\n",
      " [60 65]] [125 145]\n",
      "[[50 55]\n",
      " [60 65]\n",
      " [70 75]] [145 165]\n",
      "[[60 65]\n",
      " [70 75]\n",
      " [80 85]] [165 185]\n"
     ]
    }
   ],
   "source": [
    "# multivariate multi-step data preparation\n",
    "from numpy import array\n",
    "from numpy import hstack\n",
    "\n",
    "# split a multivariate sequence into samples\n",
    "def split_sequences(sequences, n_steps_in, n_steps_out):\n",
    "\tX, y = list(), list()\n",
    "\tfor i in range(len(sequences)):\n",
    "\t\t# find the end of this pattern\n",
    "\t\tend_ix = i + n_steps_in\n",
    "\t\tout_end_ix = end_ix + n_steps_out-1\n",
    "\t\t# check if we are beyond the dataset\n",
    "\t\tif out_end_ix > len(sequences):\n",
    "\t\t\tbreak\n",
    "\t\t# gather input and output parts of the pattern\n",
    "\t\tseq_x, seq_y = sequences[i:end_ix, :-1], sequences[end_ix-1:out_end_ix, -1]\n",
    "\t\tX.append(seq_x)\n",
    "\t\ty.append(seq_y)\n",
    "\treturn array(X), array(y)\n",
    "\n",
    "# define input sequence\n",
    "in_seq1 = array([10, 20, 30, 40, 50, 60, 70, 80, 90])\n",
    "in_seq2 = array([15, 25, 35, 45, 55, 65, 75, 85, 95])\n",
    "out_seq = array([in_seq1[i]+in_seq2[i] for i in range(len(in_seq1))])\n",
    "\n",
    "# convert to [rows, columns] structure\n",
    "in_seq1 = in_seq1.reshape((len(in_seq1), 1))\n",
    "in_seq2 = in_seq2.reshape((len(in_seq2), 1))\n",
    "out_seq = out_seq.reshape((len(out_seq), 1))\n",
    "\n",
    "# horizontally stack columns\n",
    "dataset = hstack((in_seq1, in_seq2, out_seq))\n",
    "\n",
    "# choose a number of time steps\n",
    "n_steps_in, n_steps_out = 3, 2\n",
    "\n",
    "# covert into input/output\n",
    "X, y = split_sequences(dataset, n_steps_in, n_steps_out)\n",
    "print(X.shape, y.shape)\n",
    "\n",
    "# summarize the data\n",
    "for i in range(len(X)):\n",
    "    print(X[i], y[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "c2bbf261",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "feature nums:  2\n",
      "x input:  [[[70 75]\n",
      "  [80 85]\n",
      "  [90 95]]]\n",
      "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x000001C566BFB5E8> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "[[187.14609 208.28438]]\n"
     ]
    }
   ],
   "source": [
    "# the dataset knows the number of features, e.g. 2\n",
    "n_features = X.shape[2]\n",
    "print('feature nums: ', n_features)\n",
    "\n",
    "# define model\n",
    "model = Sequential()\n",
    "model.add(LSTM(100, activation='relu', return_sequences=True, input_shape=(n_steps_in, n_features)))\n",
    "model.add(LSTM(100, activation='relu'))\n",
    "model.add(Dense(n_steps_out))\n",
    "model.compile(optimizer='adam', loss='mse')\n",
    "\n",
    "# fit model\n",
    "model.fit(X, y, epochs=200, verbose=0)\n",
    "\n",
    "# demonstrate prediction\n",
    "x_input = array([[70, 75], [80, 85], [90, 95]])\n",
    "x_input = x_input.reshape((1, n_steps_in, n_features))\n",
    "print('x input: ', x_input)\n",
    "\n",
    "yhat = model.predict(x_input, verbose=0)\n",
    "print(yhat)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cf0545b8",
   "metadata": {},
   "source": [
    "### Multiple Parallel Input and Multi-Step Output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "d42e0040",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5, 3, 3) (5, 2, 3)\n",
      "[[10 15 25]\n",
      " [20 25 45]\n",
      " [30 35 65]] [[ 40  45  85]\n",
      " [ 50  55 105]]\n",
      "[[20 25 45]\n",
      " [30 35 65]\n",
      " [40 45 85]] [[ 50  55 105]\n",
      " [ 60  65 125]]\n",
      "[[ 30  35  65]\n",
      " [ 40  45  85]\n",
      " [ 50  55 105]] [[ 60  65 125]\n",
      " [ 70  75 145]]\n",
      "[[ 40  45  85]\n",
      " [ 50  55 105]\n",
      " [ 60  65 125]] [[ 70  75 145]\n",
      " [ 80  85 165]]\n",
      "[[ 50  55 105]\n",
      " [ 60  65 125]\n",
      " [ 70  75 145]] [[ 80  85 165]\n",
      " [ 90  95 185]]\n"
     ]
    }
   ],
   "source": [
    "# multivariate multi-step data preparation\n",
    "\n",
    "# split a multivariate sequence into samples\n",
    "def split_sequences(sequences, n_steps_in, n_steps_out):\n",
    "\tX, y = list(), list()\n",
    "\tfor i in range(len(sequences)):\n",
    "\t\t# find the end of this pattern\n",
    "\t\tend_ix = i + n_steps_in\n",
    "\t\tout_end_ix = end_ix + n_steps_out\n",
    "\t\t# check if we are beyond the dataset\n",
    "\t\tif out_end_ix > len(sequences):\n",
    "\t\t\tbreak\n",
    "\t\t# gather input and output parts of the pattern\n",
    "\t\tseq_x, seq_y = sequences[i:end_ix, :], sequences[end_ix:out_end_ix, :]\n",
    "\t\tX.append(seq_x)\n",
    "\t\ty.append(seq_y)\n",
    "\treturn array(X), array(y)\n",
    "\n",
    "# define input sequence\n",
    "in_seq1 = array([10, 20, 30, 40, 50, 60, 70, 80, 90])\n",
    "in_seq2 = array([15, 25, 35, 45, 55, 65, 75, 85, 95])\n",
    "out_seq = array([in_seq1[i]+in_seq2[i] for i in range(len(in_seq1))])\n",
    "\n",
    "# convert to [rows, columns] structure\n",
    "in_seq1 = in_seq1.reshape((len(in_seq1), 1))\n",
    "in_seq2 = in_seq2.reshape((len(in_seq2), 1))\n",
    "out_seq = out_seq.reshape((len(out_seq), 1))\n",
    "\n",
    "# horizontally stack columns\n",
    "dataset = hstack((in_seq1, in_seq2, out_seq))\n",
    "\n",
    "# choose a number of time steps\n",
    "n_steps_in, n_steps_out = 3, 2\n",
    "\n",
    "# covert into input/output\n",
    "X, y = split_sequences(dataset, n_steps_in, n_steps_out)\n",
    "print(X.shape, y.shape)\n",
    "\n",
    "# summarize the data\n",
    "for i in range(len(X)):\n",
    "\tprint(X[i], y[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "025ad985",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "feature nums:  3\n",
      "x input:  [[[ 60  65 125]\n",
      "  [ 70  75 145]\n",
      "  [ 80  85 165]]]\n",
      "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x000001C5680F8798> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "[[[ 91.069855  95.898224 186.8942  ]\n",
      "  [100.35968  106.8825   206.59927 ]]]\n"
     ]
    }
   ],
   "source": [
    "# the dataset knows the number of features, e.g.3\n",
    "n_features = X.shape[2]\n",
    "print('feature nums: ', n_features)\n",
    "\n",
    "# define model\n",
    "model = Sequential()\n",
    "model.add(LSTM(200, activation='relu', input_shape=(n_steps_in, n_features)))\n",
    "model.add(RepeatVector(n_steps_out))\n",
    "\n",
    "model.add(LSTM(200, activation='relu', return_sequences=True))\n",
    "model.add(TimeDistributed(Dense(n_features)))\n",
    "model.compile(optimizer='adam', loss='mse')\n",
    "\n",
    "# fit model\n",
    "model.fit(X, y, epochs=300, verbose=0)\n",
    "\n",
    "# demonstrate prediction\n",
    "x_input = array([[60, 65, 125], [70, 75, 145], [80, 85, 165]])\n",
    "x_input = x_input.reshape((1, n_steps_in, n_features))\n",
    "print('x input: ', x_input)\n",
    "\n",
    "yhat = model.predict(x_input, verbose=0)\n",
    "print(yhat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb5defcc",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tensorflow",
   "language": "python",
   "name": "tensorflow"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
